/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright (c) 2008-2011, Salzburg Research. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 * - Redistributions of source code must retain the above copyright notice,
 * this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 * - Neither the name of the KiWi Project nor the names of its contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 * Contributor(s):
 * Thomas Kurz
 *
 * File Info:
 * User: Thomas Kurz
 * Date: 2011/02/02
 * Time: 16:29
 *
 * Project: KiWi2, kiwi-core module
 */

package at.newmedialab.sn.services.importer;

import at.newmedialab.sn.exceptions.GeoCodingFailedException;
import at.newmedialab.sn.model.geo.SNPlaceFacade;
import at.newmedialab.sn.model.importer.SNNewsItemFacade;
import at.newmedialab.sn.services.geo.GeoServiceImpl;
import kiwi.core.api.config.ConfigurationService;
import kiwi.core.api.content.ContentService;
import kiwi.core.api.facading.FacadingService;
import kiwi.core.api.importer.Importer;
import kiwi.core.api.importer.KiWiImportException;
import kiwi.core.api.transaction.Transaction;
import kiwi.core.api.transaction.TransactionService;
import kiwi.core.api.triplestore.LiteralService;
import kiwi.core.api.triplestore.ResourceService;
import kiwi.core.api.triplestore.TripleStore;
import kiwi.core.model.Constants;
import kiwi.core.model.rdf.KiWiLiteral;
import kiwi.core.model.rdf.KiWiResource;
import kiwi.core.model.rdf.KiWiTriple;
import kiwi.core.model.rdf.KiWiUriResource;
import kiwi.core.model.user.KiWiUser;
import kiwi.social.api.tagging.TaggingService;
import nu.xom.*;
import nu.xom.xslt.XSLException;
import nu.xom.xslt.XSLTransform;
import org.geonames.Toponym;
import org.slf4j.Logger;

import javax.enterprise.context.ApplicationScoped;
import javax.inject.Inject;
import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * User: Thomas Kurz
 * Date: 02.02.11
 * Time: 16:29
 */
@ApplicationScoped
public class SNImporterImpl implements Importer {

	private static final int BATCHSIZE = 50;

	@Inject
    private Logger log;

	@Inject
	private TransactionService transactionService;

	@Inject
	private TripleStore tripleStore;

	@Inject
	private ConfigurationService configurationService;

	@Inject
	private GeoServiceImpl geoService;

	@Inject
	private ResourceService resourceService;

	@Inject
	private LiteralService literalService;

	@Inject
	private ContentService contentService;

	@Inject
	private TaggingService taggingService;

	@Inject
	private FacadingService facadingService;

	private DateFormat datetimeFormat = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss");
	private DateFormat datumFormat    = new SimpleDateFormat("dd.MM.yyyy");

	//accepted types
	private static final Set<String> MIME_TYPES = new HashSet<String>(){{
		add("application/snxml+xml");
	}};

	@Override
	public String getName() {
		return "SNXML";
	}

	@Override
	public String getDescription() {
		return "Importer for parsing the Salzburger Nachrichten XML format";
	}

	@Override
	public Set<String> getAcceptTypes() {
		return MIME_TYPES;
	}

	@Override
	public int importData(URL url, String format, Set<KiWiUriResource> types, Set<KiWiResource> tags, KiWiUser user) throws KiWiImportException {
		try {
			URLConnection con = url.openConnection();

			//import from local directory
			if(url.getProtocol().equals("file")) {
				File dir = new File(url.getPath());

				Transaction transaction = transactionService.getTransaction();
				transactionService.begin(transaction);

				if(dir.exists() && dir.isDirectory()) {
					log.info("importing SNXML files from directory {}",url.getPath());

					int count = 0, failed = 0;
					Builder builder = new Builder(false);
					HashMap<String,List<SNNewsItemFacade>> map = new HashMap<String,List<SNNewsItemFacade>>();
					for(File f : dir.listFiles()) {
						if(f.getName().endsWith(".xml")) {
							try {
								SNNewsItemFacade ni = importFile(builder.build(f), f.getName(), types, tags, user);
								if(ni != null && ni.getLocationString() != null) {
									if(map.containsKey(ni.getLocationString())){
										map.get(ni.getLocationString()).add(ni);
									} else {
										List<SNNewsItemFacade> l = new ArrayList<SNNewsItemFacade>();
										l.add(ni);
										map.put(ni.getLocationString(),l);
									}
								 	count++;
								} else {
									failed++;
								}
							} catch (SNImportException e) {
								log.error(e.getMessage());
								failed++;
							} catch (ValidityException e) {
								log.error("error while validating SNXML file {}: {}", f.getName(), e.getMessage());
								failed++;
							} catch (ParsingException e) {
								log.error("error while parsing SNXML file {}: {}", f.getName(), e.getMessage());
								failed++;
							} catch(FileNotFoundException ex) {
								log.error("file not found: {}",f.getName());
								failed++;
							}
							if(count % 50 == 0) {
								//geolocate
								for(String key : map.keySet()) {
									geolocate(key,map.get(key));
								}
								map = new HashMap<String,List<SNNewsItemFacade>>();
								//end transaction and start new
								transactionService.commit(transaction);
								transaction = transactionService.getTransaction();
								transactionService.begin(transaction);
							}
						}
					}
					//geolocate
					for(String key : map.keySet()) {
						geolocate(key,map.get(key));
					}
					transactionService.commit(transaction);
					log.info("imported {} SN articles, {} imports failed",count,failed);
					return count;
				} else {
					log.info("importing possible SNXML file {}",url.getPath());

					// a possible single SN News article; import from input stream
					try {
						SNNewsItemFacade ni = importFile(new Builder(false).build(dir), dir.getName(), types, tags, user);
						if(ni != null && ni.getLocationString() != null) {
							List<SNNewsItemFacade> l = new ArrayList<SNNewsItemFacade>();
							l.add(ni);
							geolocate(ni.getLocationString(), l);
							return 1;
						} else
							return 0;
					} catch (ValidityException e) {
						log.error("error while validating SNXML file {}: {}", dir.getName(), e.getMessage());
					} catch (ParsingException e) {
						log.error("error while parsing SNXML file {}: {}", dir.getName(), e.getMessage());
					} catch (SNImportException e) {
						log.warn("SNXML file {} skipped: {}", dir.getName(),e.getMessage());
					}
					return 0;
				}
			} else if(con.getContentType() != null && con.getContentType().equals("application/xml")) {
				// a possible SN News article given by url; import from input stream
				return importData(con.getInputStream(),format,types,tags,user);
			} else {
				log.error("unsupported content type and prototcol for URL {}: {}", url.toString(), con.getContentType());
			}
		} catch (IOException e) {
			log.error(e.getLocalizedMessage());
		}
		return 0;
	}

	@Override
	public int importData(InputStream is, String filename, Set<KiWiUriResource> types, Set<KiWiResource> tags, KiWiUser user) throws KiWiImportException {
		try {
			Transaction t = transactionService.getTransaction();
			transactionService.begin(t);
			Document doc = new Builder(false).build(is);
			SNNewsItemFacade ni = importFile(doc, filename, types, tags, user);
			if(ni != null && ni.getLocationString() != null) {
				List<SNNewsItemFacade> l = new ArrayList<SNNewsItemFacade>();
				l.add(ni);
				geolocate(ni.getLocationString(), l);
				transactionService.commit(t);
				return 1;
			} else {
				transactionService.commit(t);
				return 0;
			}
		} catch (ValidityException e) {
			log.error("error parsing XML file {} from input stream",filename,e);
		} catch (ParsingException e) {
			log.error("error parsing XML file {} from input stream",filename,e);
		} catch (IOException e) {
			log.error("error parsing XML file {} from input stream",filename,e);
		} catch (SNImportException e) {
			log.warn("SNXML file {} skipped: {}", filename,e.getMessage());
		}
		return 0;
	}

	@Override
	public int importData(Reader reader, String filename, Set<KiWiUriResource> types, Set<KiWiResource> tags, KiWiUser user) throws KiWiImportException {
		try {
			Transaction t = transactionService.getTransaction();
			transactionService.begin(t);
			Document doc = new Builder(false).build(reader);
			SNNewsItemFacade ni = importFile(doc, filename, types, tags, user);
			if(ni != null && ni.getLocationString() != null) {
				List<SNNewsItemFacade> l = new ArrayList<SNNewsItemFacade>();
				l.add(ni);
				geolocate(ni.getLocationString(), l);
				transactionService.commit(t);
				return 1;
			} else {
				transactionService.commit(t);
				return 0;
			}
		} catch (ValidityException e) {
			log.error("error parsing XML file {} from input stream",filename,e);
		} catch (ParsingException e) {
			log.error("error parsing XML file {} from input stream",filename,e);
		} catch (IOException e) {
			log.error("error parsing XML file {} from input stream",filename,e);
		} catch (SNImportException e) {
			log.warn("SNXML file {} skipped: {}", filename,e.getMessage());
		}
		return 0;
	}

	private SNNewsItemFacade importFile(Document doc, String filename, Set<KiWiUriResource> types, Set<KiWiResource> tags, KiWiUser user) throws SNImportException {
		String uri = configurationService.getBaseUri() + "resource/"+filename;
		// check whether file has already been loaded
		if(resourceService.getUriResource(uri) != null) {
			log.info("skipping file {}",filename);
			return null;
		}
		log.info("processing file {}",filename);

		try {
			String title = null, ressort = null, geo = null, ecid = null, summary = null, con = null;
			Set<String> keywords = new HashSet<String>();
			Date date = null;
			Document content = null;
			Nodes nl;

			// look for ecid
			nl = doc.query("/SNXML/ECID/text()");
			if(nl.size() > 0) {
				ecid = ((Text)nl.get(0)).getValue();
			} else {
				log.debug("file {} skipped as it does not contain a ecid identifier",filename);
			}


			// look for title
			nl = doc.query("/SNXML/ARTIKEL/INHALT/TITEL/text()");
			if(nl.size() > 0) {
				title = ((Text)nl.get(0)).getValue();
				if(title.length() > 200) {
					title = title.substring(0,200);
				}
			} else {
				log.error("file {} skipped as it does not contain a title",filename);
				throw new SNImportException("no title");
			}

			//look for short description
			nl = doc.query("/SNXML/ARTIKEL/INHALT/VORSPANN/text()");
			if(nl.size() > 0) {
				summary = ((Text)nl.get(0)).getValue();
			} else {
				log.error("file {} skipped as it does not contain a summary",filename);
				throw new SNImportException("no short description");
			}

			//get content
			nl = doc.query("/SNXML/ARTIKEL/INHALT");
			if(nl.size() > 0) {
				con = nl.get(0).toXML();
			} else {
				log.warn("file {} does not contain content",filename);
			}

			// look for geo tag
			nl = doc.query("/SNXML/ARTIKEL/GEO/text()");
			if(nl.size() > 0) {
				geo = ((Text)nl.get(0)).getValue();
				if(geo.contains("/") ) {
					String[] s = geo.split("/");
					geo = s[0].toLowerCase().replace('.', ' ').trim();
				} else {
					geo = geo.toLowerCase().replace('.', ' ').trim();
				}

				// normalize strange patterns where the location name actually occurs twice...
				Pattern p = Pattern.compile("(\\S+)\\s+\\1");
				Matcher m = p.matcher(geo);
				if(m.matches()) {
					geo = m.group(1);
				}
			}
			log.debug("geo-location: {}",geo);

			// look for ressort tag
			nl = doc.query("/SNXML/ARTIKEL/RESSORT/text()");
			if(nl.size() > 0) {
				ressort = ((Text)nl.get(0)).getValue().toLowerCase().trim();
				keywords.add(ressort);
			}

			// look for keyword tags
			nl = doc.query("/SNXML/ARTIKEL/KEYWORD/text()");
			for(int i = 0; i < nl.size(); i++) {
				keywords.add(((Text)nl.get(0)).getValue().toLowerCase().trim());
			}

			// look for datum and timedata tags
			nl = doc.query("/SNXML/DATUM/text()");
			if(nl.size() > 0) {
				try {
					date = datumFormat.parse(((Text)nl.get(0)).getValue());
				} catch (ParseException e) {
					log.warn("error while parsing date: {}",((Text)nl.get(0)).getValue());
				}
			}
			nl = doc.query("/SNXML/ARTIKEL/TIMEDATE/text()");
			if(nl.size() > 0) {
				try {
					date = datetimeFormat.parse(((Text)nl.get(0)).getValue());
				} catch (ParseException e) {
					log.warn("error while parsing date: {}",((Text)nl.get(0)).getValue());
				}
			}

			// now look for content and transform it to HTML
			try {
				// we use the interface as classloader reference, since the stateless bean has a different classloader
				Document stylesheet = new Builder(false).build(SNNewsItemFacade.class.getResourceAsStream("snhtml.xsl"));
				XSLTransform transform = new XSLTransform(stylesheet);
				Nodes output = transform.transform(doc);
				content = XSLTransform.toDocument(output);
			} catch (XSLException e) {
				log.error("error while transforming XML content of file {}: {}",filename,e.getMessage());
			}

			// if we have come so far, we can create a new content item for the imported data
			KiWiUriResource resource = resourceService.createUriResource(uri);
			resource.setCreator(user);

			SNNewsItemFacade newsItem = facadingService.createFacade(resource,SNNewsItemFacade.class);

			if(date != null) {
				resource.setCreated(date);
			}
			if(content != null) {
				contentService.setContent(resource,content.toXML().getBytes("UTF8"),"text/html");
			}

			if(summary != null) {
				newsItem.setSummary(summary);
			}

			if(ecid != null) {
				newsItem.setECID(ecid);
			}

			if(title!=null) {
				newsItem.setTitle(title);
			}

			// for each of the keywords, do the same
			for(String keyword : keywords) {
				if(!"".equals(keyword)) {
					//create Tagging
					taggingService.createTagging(resource,keyword,user);
					//TODO category, subcategory
				}
			}

			if(geo!=null) {
				newsItem.setLocationString(geo);
			}

			if(ressort!=null) {
				newsItem.setRessort(ressort);
			}

			return newsItem;

		} catch (Exception e) {
			log.error("could not import file:"+e.getMessage());
			return null;
		}

	}

	private void geolocate(String name, List<SNNewsItemFacade> resources) {
		if (name != null) {
			//query for resource with locationString==name
			KiWiResource location = null;
			KiWiLiteral object = literalService.getLiteral(name);
			KiWiUriResource property = resourceService.getUriResource(Constants.NS_FCP_CORE + "loc");
			if (object != null && property != null) {
				Set<KiWiTriple> t = tripleStore.listTriples(null, property, object, Collections.<KiWiUriResource>emptySet(), false, 0, 1);
				if (!t.isEmpty()) {
					KiWiResource r = t.iterator().next().getSubject();
					SNNewsItemFacade ni = facadingService.createFacade(r, SNNewsItemFacade.class);
					location = ni.getLocation();
				}
			}
			if (location == null) {
				try {
					Toponym top = geoService.getToponym(name);
					if(top!=null) {
						String internal_url = configurationService.getBaseUri()+"resource/geonames_"+top.getGeoNameId();
						location = resourceService.getUriResource(internal_url);
						if(location == null) {
							location = resourceService.createUriResource(internal_url);
							SNPlaceFacade place = facadingService.createFacade(location,SNPlaceFacade.class);
							String geonames_url = "http://sws.geonames.org/"+top.getGeoNameId()+"/";
							KiWiResource orig = resourceService.getUriResource(geonames_url);
							if(orig==null) {
								orig=resourceService.createUriResource(geonames_url);
							}
							//set place
							place.setName(top.getName());
							place.setLatitude(top.getLatitude());
							place.setLongitude(top.getLongitude());
							place.setCountryCode(top.getCountryCode());
							place.setOriginal(orig);
						}
					}
				} catch (GeoCodingFailedException e) {
					log.error(e.getMessage());
				}
			}
			if(location != null) {
				for(SNNewsItemFacade nif : resources) {
					nif.setLocation(location);
				}
			}
		}
	}
}
